# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from scrapy.selector import Selector
from urlparse import urljoin
import re
from mobilenumber.items import MobilenumberItem

class NhrenSpider(scrapy.Spider):
    name = "nhren"
    allowed_domains = ["nhren.com"]
    start_urls = [
        'http://nhren.com/thread-935698-1-1.html'
        #'http://nhren.com/forum-482-1.html',
    ]

    def start_requests(self):
        yield Request(url='http://nhren.com/thread-935698-1-1.html', dont_filter=True, callback=self.parse)


    def parse(self, response):
        if u'BBS会员登陆后才可以查看！' in response.body.decode('gbk'):
            yield scrapy.FormRequest.from_response(
                response,
                formdata={'username': 'liugehao', 'password': 'liuyou'},
                callback=self.parse,

            )
        else:
            """
            yield Request(url='http://nhren.com/thread-935698-1-1.html', dont_filter=True, callback=self.parse_page)
            """
            for url in [
                'http://nhren.com/forum-482-1.html',
                'http://nhren.com/forum-693-1.html',

            ]:
                yield Request(url=url, callback=self.parse_list, dont_filter=True)



    def parse_list(self, response):
        sel = Selector(response)
        for url in sel.xpath('//*/tr/th/a/@href').extract():
            yield Request(url=urljoin(response.url, url), callback=self.parse_page)

        for num in sel.xpath('//*[@id="fd_page_top"]/div/a/text()').re('\d+'):
            yield Request(url=re.sub('-\d+\.html', '-%s.html' % num, response.url), callback=self.parse_list)

    def parse_page(self, response):
        for mobile in list(set(re.findall('1[34578]\d{9}', response.body))):
            print mobile
            mn = MobilenumberItem()
            mn['MobileNumber'] = mobile
            yield mn
        if 'sortmessag' in response.body:
            url =re.search('"ajaxget\(\'(.*?)\'', response.body).group(1)
            yield Request(url=urljoin(response.url, url), callback=self.parse_page)





